﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using StockFinder.Model;
using System.IO;
using log4net;
using System.Net;
using System.Text.RegularExpressions;
using System.Diagnostics;

namespace StockFinder.Services.Engines.Yahoo
{
    public class YahooSymbolIndexUrlExtractEngine : ISymbolIndexUrlExtractor
    {
        private static readonly ILog _Log = LogManager.GetLogger(typeof(YahooSymbolIndexUrlExtractEngine));

        const string TOP_LEVEL_REGEX_PATTERN = "<td nowrap><a href=([\\w\\s/-]+\\.html)>([\\w-]+)</a></td>";
        const string SECOND_LEVEL_REGEX_PATTERN = "<td nowrap[=\"\\w\\s]*><a href=([\"\\w\\s/-]+\\.html[\"]?)>([\\w\\s-]+)[<>\\(\\)\\w/]*</a></td>";
        const string BASE_URL = "http://biz.yahoo.com";
        const string INDEX_URL = "http://biz.yahoo.com/i/";

        #region ISymbolIndexUrlExtractor Members

        public event Action<string> LogUpdate;

        public List<IndexUrl> GetUrls()
        {
            List<IndexUrl> indexUrls = new List<IndexUrl>();
         
            #region Get Main Index Source

            bool loadedBaseIndexSourceOk = false;

           OnLogUpdate("Making base request...");

            //make the request
            HttpWebRequest webRequest = HttpWebRequest.Create(INDEX_URL) as HttpWebRequest;

            HttpWebResponse webResponse = null;

            try
            {
                //try get the response
                webResponse = webRequest.GetResponse() as HttpWebResponse;
            }
            catch (Exception exception)
            {
                //error occured
                OnLogUpdate(string.Format("Error getting base index url page source: {0}", exception.ToString()));
            }

            //check valid webresponse from base index url
            if (webResponse != null)
            {
                string responsePageSource = null;

                try
                {
                    //create the reader to extract the page source
                    using (StreamReader streamReader = new StreamReader(webResponse.GetResponseStream()))
                    {
                        OnLogUpdate("Reading page source...");

                        //get the page source
                        responsePageSource = streamReader.ReadToEnd();
                    }
                }
                catch (Exception exception)
                {
                    //something happened either getting respone stream or getting page source
                    OnLogUpdate(string.Format("Error occurred reading page response: {0}", exception.ToString()));
                }

                //check there is valid page source
                if (string.IsNullOrEmpty(responsePageSource))
                {
                    OnLogUpdate("Base Index page source is either null or empty; exiting");
                }
                else
                {
                    #region Get Top Level Indexes e.g. 'G'

                    webResponse = null;
                    webRequest = null;

                    /* extract all top levels */

                    //tidy up page source
                    responsePageSource = responsePageSource.Replace("\n", " ");

                    OnLogUpdate("Finding Urls in source");

                    //regex the indexs
                    MatchCollection matches = Regex.Matches(responsePageSource, TOP_LEVEL_REGEX_PATTERN, RegexOptions.Multiline);

                    OnLogUpdate(string.Format("Found {0} matches", matches.Count));

                    indexUrls = new List<IndexUrl>();

                    OnLogUpdate("Looping each match");

                    foreach (Match match in matches)
                    {
                        if (match.Groups.Count == 3)
                        {
                            IndexUrl topLevelIndex = new IndexUrl();
                            topLevelIndex.Url = string.Format("{0}{1}", BASE_URL, match.Groups[1].Value);
                            topLevelIndex.Title = match.Groups[2].Value.Trim();

                            OnLogUpdate(string.Format("Top Level Index Found: {0} {1}", topLevelIndex.Url, topLevelIndex.Title));

                            indexUrls.Add(topLevelIndex);                            
                        }
                    }

                    OnLogUpdate(string.Format("Found {0} indexs", indexUrls.Count));

                    #endregion

                    #region Get second level indexes e.g. 'Ga' to 'Gd'

                    OnLogUpdate("Finding second level indexes...");

                    foreach (IndexUrl topLevelIndex in indexUrls)
                    {
                        OnLogUpdate(string.Format("Making second level request for '{0}'", topLevelIndex.Url));

                        webRequest = HttpWebRequest.Create(topLevelIndex.Url) as HttpWebRequest;

                        webResponse = webRequest.GetResponse() as HttpWebResponse;

                        using (StreamReader streamReader = new StreamReader(webResponse.GetResponseStream()))
                        {
                            OnLogUpdate("Reading page source...");

                            //get the page source
                            responsePageSource = streamReader.ReadToEnd();
                        }

                        /* extract all second levels on page */

                        responsePageSource = responsePageSource.Replace("\n", " ");

                        matches = Regex.Matches(responsePageSource, SECOND_LEVEL_REGEX_PATTERN, RegexOptions.Multiline);

                        OnLogUpdate(string.Format("Found {0} matches", matches.Count));

                        topLevelIndex.Children = new List<IndexUrl>();

                        foreach (Match match in matches)
                        {
                            if (match.Groups.Count == 3)
                            {
                                string url = string.Format("{0}{1}", BASE_URL, match.Groups[1].Value);
                                string title = match.Groups[2].Value.Trim();

                                OnLogUpdate(string.Format("SecondLevel: {0} {1}", url, title));

                                //have we accidently found a top level?
                                if (indexUrls.Exists(delegate(IndexUrl t) { return title == t.Title; }))
                                {
                                    OnLogUpdate("Ignoring because already a top level");
                                }
                                else
                                {
                                    //no so create a secondlevel component
                                    IndexUrl secondLevelIndex = new IndexUrl();
                                    secondLevelIndex.Url = url;
                                    secondLevelIndex.Title = title.Trim();

                                    OnLogUpdate(string.Format("Second Level Index Found: {0} {1}", secondLevelIndex.Url, secondLevelIndex.Title));

                                    topLevelIndex.Children.Add(secondLevelIndex);
                                    secondLevelIndex.Parent = topLevelIndex;
                                }
                            }
                        }

                    }

                    OnLogUpdate("Completed...");

                    #endregion

                    #region Generating url string list

                    List<string> urls = new List<string>();

                    foreach (IndexUrl topLevelIndex in indexUrls)
                    {
                        urls.Add(topLevelIndex.Url);

                        foreach (IndexUrl secondLevelIndex in topLevelIndex.Children)
                        {
                            urls.Add(secondLevelIndex.Url);
                        }
                    }

                    _Log.InfoFormat("Generated {0} urls", urls.Count);

                    #endregion            
                }
            }            

            #endregion

            

            return indexUrls;
        }

        public void Stop()
        {
            throw new NotImplementedException();
        }

        #endregion

        protected virtual void OnLogUpdate(string logMessage)
        {
            if (LogUpdate != null)
            {
                LogUpdate(logMessage);
            }
        }
    }
}
